package com.shujia.spark.core

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object Demo7Union {
  def main(args: Array[String]): Unit = {
    //1、创建环境
    val conf = new SparkConf()
    //4:4核
    conf.setMaster("local")
    conf.setAppName("group")
    val sc = new SparkContext(conf)

    val rdd1: RDD[String] = sc.textFile("data/students.txt")
    val rdd2: RDD[String] = sc.textFile("data/students.txt")

    //union:相当于sql中的union all
    val unionRDD: RDD[String] = rdd1.union(rdd2)

    val count: Long = unionRDD.count()
    println(count)


    //distinct: 会产生shuffle,需要将相同的数据拉取到同一个reduce中才能去重
    val distinctRDD: RDD[String] = unionRDD.distinct()
    println(distinctRDD.count())

    while (true){}

  }
}
